import urllib.robotparser
from urllib.parse import urlparse
from urllib.request import urlopen

if __name__ == "__main__":
    rp = urllib.robotparser.RobotFileParser()
    rp.set_url("http://www.musi-cal.com/robots.txt")
    rp.read()
    rrate = rp.request_rate("*")
    rrate.requests
    rrate.seconds
    rp.crawl_delay("*")
    rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco")
    rp.can_fetch("*", "http://www.musi-cal.com/")